#coding=utf-8

'''

      分析

   ~35k posts, costs ~ 45m

'''
#import jieba
#import jieba.posseg as pseg
from optparse import OptionParser
import os, sys
import json
import re

sys.path.append(os.path.abspath(os.path.dirname(os.path.join(__file__, '../../'))))
from ripper.syncconfig import sync_db_host, sync_db_user, sync_db_pwd, sync_db_name

import MySQLdb
from datetime import date
import gc
#gc.disable()

# tag输入文件: tag_type
tagsource = {'codes.txt': 1, 'names.txt': 2}

print 'connecting...', sync_db_host
db = MySQLdb.connect(sync_db_host, sync_db_user, sync_db_pwd, sync_db_name)
print 'connected'
db.set_character_set('utf8')

class BrandTagging(object):
    '''Count tags against post title or content'''

    def get_conn(self):
        print 'connecting...', sync_db_host
        _db = MySQLdb.connect(sync_db_host, sync_db_user, sync_db_pwd, sync_db_name)
        print 'connected'
        _db.set_character_set('utf8')
        return _db
   
    def _load_post_titles(self):

        sql = 'select title, pid from post where fid in (4, 230, 143) '        
        all = []  
        cur = db.cursor()
        cur.execute(sql)
       # result =  cur.fetchone()
       # while result != None:
       #     title, pid = result
       #     yield title, pid
       #     result = cur.fetchone()
        result =  cur.fetchall()
        for r in result:
            title, pid = r
            all.append( (title, pid) )
        cur.close()
        return all

    def count_tag(self):
        tagdict = {}
        print 'loading tags...'
        for sourcefile, tagtype in tagsource.items():
            with open(sourcefile, 'r') as ff:
                for line in ff.read().split('\n'):
                    line = line.strip()
                    if line == '': continue
                    if tagtype == 1:
                        # 匹配code, 两侧不能为字母
                        pt = '.*?[^a-zA-Z]*?' + line + '[^a-zA-Z]'
                        tagdict.setdefault(tagtype, {})[(re.compile(pt), line)] = [0, set() ]
                    else:
                        tagdict.setdefault(tagtype, {})[line.lower()] = [0, set() ]
        
        print 'loading post titles....'
        all = self._load_post_titles()
        print 'analyzing ...'
        for post in all: 
            title, pid = post
            title = title.upper()
            #print '..........', title
            for tagtype, tagnamedict in tagdict.iteritems():
                for tagname in tagnamedict.iterkeys():
                    if tagtype == 1:
                        tagname_re, _tagname = tagname
                        #if None != tagname_re.match(title):
                        if _tagname in title or \
                                _tagname.replace(' ', '') in title:
                            #print '0000000000', title
                            tagnamedict[tagname][0] += 1
                            tagnamedict[tagname][1].add(pid)
                    elif tagtype == 2:
                        # 匹配人名
                        if tagname in title or \
                                tagname.replace(' ', '') in title:
                            tagnamedict[tagname][0] += 1
                            tagnamedict[tagname][1].add(pid)
        
        self.transfer_tag_data(tagdict)


    def transfer_tag_data(self, tagdict):
        '''上传tag数据'''
        sqltpl = 'insert into tag (tag_key, tag_name, tag_type, tag_count) values (%s,%s,%s,%s) '
        sqltpl_rela = 'insert into tag_rela (tag_key, pid, tag_type) values (%s,%s,%s) '
        tags = []
        relas = []
        print 'preparing data...'
        for tagtype , tagnamedict in tagdict.iteritems():
            for tagname, vals in tagnamedict.iteritems():
                count, pids = vals
                if count != 0:
                    if tagtype == 1:
                        tagname = tagname[1]
                    tagkey = '%s_%d' % (tagname, tagtype)
                    tags.append([ tagkey, 
                                    tagname, 
                                    int(tagtype),
                                    int(count)])
                    for pid in pids : 
                        rela = [tagkey, pid, tagtype]
                        relas.append(rela)
        print 'inserting...'
        db = self.get_conn()
        cur = db.cursor()
        cur.execute('SET autocommit=0;')
        cur.execute('truncate table tag')
        cur.execute('truncate table tag_rela')
        cur.executemany(sqltpl, tags)
        cur.executemany(sqltpl_rela, relas)
        print 'commiting...'
        db.commit()
        cur.close()


class MenuItemCounter(object):

    def __init__(self):
        tags = []
        for fid in self.loadAllFids():
            for cat, count in self.gettags(fid):
                data = [fid, cat, count]
                tags.append(data)

        cur = db.cursor()
        cur.execute('SET autocommit=0;')
        cur.execute('truncate table menu')
        sql = 'insert into menu (fid, cat, count) values(%s, %s, %s)'
        cur.executemany(sql, tags)
        print 'commiting...'
        db.commit()
        cur.close()

    def loadAllFids(self):
        sql = 'select distinct(fid) from post'
        cur = db.cursor()
        cur.execute(sql)
        all = []
        result =  cur.fetchall()
        for fid in result:
            all.append(fid[0]) 
        cur.close()
        return all

    def gettags(self, fid):
        sql = 'select cat, count(1) from post where fid=%s group by cat' %str(fid)
        cur = db.cursor()
        cur.execute(sql)
        all = []
        result =  cur.fetchall()
        for result in result:
            cat, ct = result
            if cat == '':
                cat = u'无分类'
                
            all.append((cat, ct)) 
        cur.close()
        return all

def stat():
    sql = '''select sum(a.sz) from ( select pid,size, CONVERT(SUBSTRING_INDEX(size,' ',1),UNSIGNED INTEGER) as sz from torrent where size like '%MB%'  ) a'''
    cur = db.cursor()
    cur.execute(sql)
    all_mb = cur.fetchone()[0]
    sql = '''select sum(a.sz) from ( select pid,size, CONVERT(SUBSTRING_INDEX(size,' ',1),UNSIGNED INTEGER) as sz from torrent where size like '%GB%' ) a'''
    cur = db.cursor()
    cur.execute(sql)
    all_gb = cur.fetchone()[0]
    all_tb = all_mb/1024/1024 + all_gb/1024

if __name__ == '__main__':
    usage = u"usage: %prog 无参数:同步tag, menu:fid->cat to menu"
    parser = OptionParser(usage)
    (options, args) = parser.parse_args()
    if 'menu' in args:
        print 'counting menu items...'
        MenuItemCounter()
    else:
        print 'counting name and codes...'
        BrandTagging().count_tag()

    
